#!/bin/bash
###################################################################
# @Description : Kubernetes Network Benchmark
# @Author      : Alexis Ducastel <alexis@infrabuilder.com>
# @License     : MIT
###################################################################

#==============================================================================
# Utility functions
#==============================================================================

[ "$(tput colors)" -gt 0 ] && COLOR="true" || COLOR="false"
function color { 
	$COLOR || return 0
	color="0"
	case $1 in
		normal) color="0" ;;
		rst) color="0" ;;
		red) color="31" ;;
		green) color="32" ;;
		yellow) color="33" ;;
		blue) color="34" ;;
		magenta) color="35" ;;
		cyan) color="36" ;;
		lightred) color="91" ;;
		lightgreen) color="92" ;;
		lightyellow) color="93" ;;
		lightblue) color="94" ;;
		lightmagenta) color="95" ;;
		lightcyan) color="96" ;;
		white) color="97" ;;
		*) color="0" ;;
	esac
	echo -e "\033[0;${color}m"
}
function logdate { date "+%Y-%m-%d %H:%M:%S"; }
function fatal { echo "$(logdate) $(color red)[FATAL]$(color normal) $@" >&2; cleanandexit 1; }
function err { echo "$(logdate) $(color lightred)[ERROR]$(color normal) $@" >&2; }
function warn { [$DEBUG_LEVEL -ge 1 ] && echo "$(logdate) $(color yellow)[WARNING]$(color normal) $@" >&2; }
function info { [ $DEBUG_LEVEL -ge 2 ] && echo "$(logdate) $(color cyan)[INFO]$(color normal) $@" >&2; }
function debug { [ $DEBUG_LEVEL -ge 3 ] && echo "$(logdate) $(color lightcyan)[DEBUG]$(color normal) $@" >&2; }

function now { date +%s; }
function cleanandexit {
	# Cleaning Kubernetes resources
	for i in $RESSOURCE_TO_CLEAN_BEFORE_EXIT
	do
		$DEBUG && debug "Deleting resource $i"
		kubectl delete $NAMESPACEOPT $i --wait=false >/dev/null
	done
	
	# Cleaning temp directory
	if [ "$CLEANDIR" = "true" ] 
	then
		[ ! -z "$DATADIR" ] && [ -d $DATADIR ] && touch $DATADIR/something && rm $DATADIR/* && rmdir $DATADIR
	else
		info "Keeping data directory, you will have to clean it manually."
		info "$DATADIR"
	fi

	exit $1
}

# Example : waitpod podname Running 60
function waitpod {
	POD=$1
	PHASE=$2
	TIMEOUT=$3
	TMAX=$(( $(now) + $TIMEOUT ))
	$DEBUG && debug "Waiting for pod $POD to be $PHASE until $TMAX"
	while [ "$(now)" -lt "$TMAX" ]
	do
		CURRENTPHASE=$(kubectl get --request-timeout 2s $NAMESPACEOPT pod $POD -o jsonpath={.status.phase})
		$DEBUG && debug "[$(now)/$TMAX] Pod $POD is in phase $CURRENTPHASE, waiting for $PHASE"
		[ "$CURRENTPHASE" = "$PHASE" ] && return 0
		sleep 1
	done
	return 1
}

#==============================================================================
# Default config
#==============================================================================

VERSION=1.4.1
GENERATOR="knb"
BENCHMARK_DATE=$(date -u "+%Y-%m-%d %H:%M:%S")
DEBUG=false
DEBUG_LEVEL=0
SERVER_NODE=""
CLIENT_NODE=""
EXECID="$$"
BENCHMARK_RUN_NAME="knb-$EXECID"
NAMESPACEOPT=""
POD_WAIT_TIMEOUT="30"
BENCHMARK_DURATION="10"
SOCKET_BUFFER_SIZE="auto"
RESSOURCE_TO_CLEAN_BEFORE_EXIT=""
OUTPUT_FORMAT="text"
FILENAME="/dev/stdout"
CLEANDIR="true"
FROM_DATA="false"

RUN_TEST_IDLE="true"
RUN_TEST_P2P_TCP="true"
RUN_TEST_P2P_UDP="true"
RUN_TEST_P2S_TCP="true"
RUN_TEST_P2S_UDP="true"

export LC_ALL=C # Trick to no depend on locale
BIN_AWK="awk"

#==============================================================================
# Core functions
#==============================================================================

function detect-pod-failure-reason-and-fatal {
	POD_NAME=$1
	shift

	CURRENTPHASE=$(kubectl get --request-timeout 2s $NAMESPACEOPT pod $POD_NAME -o jsonpath={.status.phase})
	$DEBUG && debug "Trying to detect pod failure for pod '$POD_NAME' in phase '$CURRENTPHASE'"
	case $CURRENTPHASE in
		"Pending")
			err "Pod $POD_NAME is still in Pending state, please check node name"
			;;
		"Failed")
			# Is it a window size problem in iperf ?
			kubectl logs $NAMESPACEOPT $POD_NAME \
				| grep "iperf3: error - unable to write to stream socket: Message too large" > /dev/null \
				&& err "Iperf message was too large, you should try a lower value with flag '-sbs' (current value is : $SOCKET_BUFFER_SIZE)" && return 0

			;;
		*) 
			err "Unknown pod phase '$CURRENTPHASE' for pod '$POD_NAME'" 
			;;
	esac

	fatal $@
}

function usage {
	cat <<-EOF

	knb is a network benchmark tool for Kubernetes CNI

	There are two modes : 
	- benchmark mode : will actually run benchmark on a cluster
	- from data mode : read data generated by previous benchmark with "-o data" flag
	
	=====[ Benchmark mode ]====================================================

	 Mandatory flags :

	    -cn <nodename>
	    --client-node <nodename>    : Define kubernetes node name that will host the client part

	    -sn <nodename>
	    --server-node <nodename>    : Define kubernetes node name that will host the server part
	
	 Optionnal flags :
	    -d <time-in-scd>
	    --duration <time-in-scd>    : Set the benchmark duration for each test in seconds (Default ${BENCHMARK_DURATION})

	    -k
	    --keep                      : Keep data directory instead of cleaning it (tmp dir that contains raw benchmark data)

	    -n <namespace>
	    --namespace <namespace>     : Set the target kubernetes namespace

	    --name <name>               : Set the name of this benchmark run

	    -ot <testlist>
	    --only-tests <testlist>     : Only run a subset of benchmark tests, comma separated (Ex: -ot tcp,idle)
	                                  Possible values: all, tcp, udp, p2p, p2s , p2ptcp, p2pudp, p2stcp, p2sudp, idle

	    -sbs <size>
	    --socket-buffer-size <size> : Set the UDP socket buffer size with unit, or 'auto'. ex: '256K' (Default: $SOCKET_BUFFER_SIZE)

	    -t <time-in-scd>
	    --timeout <time-in-scd>     : Set the pod ready wait timeout in seconds (Default ${POD_WAIT_TIMEOUT})

	=====[ From Data mode ]====================================================

	Mandatory flags :
	    -fd <path>
	    --from-data <path>          : Define the path to the data to read from

	=====[ Common optionnal flags ]============================================

	    --debug                     : Set the debug level to "debug"

	    -dl <level>
	    --debug-level <level>       : Set the debug level
	                                  Possible values: standard, warn, info, debug
	    
	    -f <filepath>
	    --file <filepath>           : Set the output file

	    -h
	    --help                      : Display this help message

	    -o <format>
	    --output <format>           : Set the output format
	                                  Possible values: text, yaml, json, data, ibdbench
	    -v
	    --verbose                   : Activate the verbose mode by setting debug-level to 'info'
		
	    -V
	    --version                   : Show current script version

	=====[ Examples ]==========================================================
	
	  Simple benchmark from "node1" to "node2" in verbose mode
	  -------------------------------------------------------------------------
	  | knb -v -cn node1 -sn node2                                            |
	  -------------------------------------------------------------------------

	  Benchmark from "nA" to "nB" with data saved in file "mybench.knbdata"
	  -------------------------------------------------------------------------
	  | knb -cn nA -sn nB -o data -f mybench.knbdata                          |
	  -------------------------------------------------------------------------

	  Generate report in json from previous benchmark file "mybench.knbdata"
	  -------------------------------------------------------------------------
	  | knb -fd mybench.knbdata -o json                                       |
	  -------------------------------------------------------------------------

	  Run only idle and tcp benchmark :
	  -------------------------------------------------------------------------
	  | knb -cn clientnode -sn servernode -ot idle,tcp                        |
	  -------------------------------------------------------------------------
	  
EOF
}

function run-client {
	POD_NAME=$1
	TARGET=$2

	SOCKET_BUFFER_FLAG=""
	if [ "$SOCKET_BUFFER_SIZE" = "auto" ]
	then
		SOCKET_BUFFER_FLAG=""
	else
		SOCKET_BUFFER_FLAG="-w $SOCKET_BUFFER_SIZE"
	fi

	CMD=""
	case $3 in
		idle) CMD="sleep $BENCHMARK_DURATION; echo 0 0 0 0 0 0 0 receiver" ;;
		udp) CMD="iperf3 -u -b 0 -c $TARGET -O 1 $SOCKET_BUFFER_FLAG -f m -t $BENCHMARK_DURATION" ;;
		tcp) CMD="iperf3 -c $TARGET -O 1 -f m -t $BENCHMARK_DURATION" ;;
		*) fatal "Unknown benchmark type '$2'" ;;
	esac

	info "Starting pod $POD_NAME on node $CLIENT_NODE"
	cat <<-EOF | kubectl apply $NAMESPACEOPT -f - >/dev/null|| fatal "Cannot create pod $POD_NAME"
	apiVersion: v1
	kind: Pod
	metadata:
	  labels:
	    app: $POD_NAME
	  name: $POD_NAME
	spec:
	  containers:
	  - name: iperf
	    image: infrabuilder/bench-iperf3
	    args:
	    - /bin/sh
	    - -c
	    - $CMD
	  nodeSelector:
	    kubernetes.io/hostname: $CLIENT_NODE
	  restartPolicy: Never
	EOF

	RESSOURCE_TO_CLEAN_BEFORE_EXIT="$RESSOURCE_TO_CLEAN_BEFORE_EXIT pod/$POD_NAME"

	# Waiting for Pod to be Running
	kubectl wait $NAMESPACEOPT --for=condition=Ready pod/$POD_NAME \
		--timeout=${POD_WAIT_TIMEOUT}s >/dev/null 2>/dev/null \
		|| detect-pod-failure-reason-and-fatal $POD_NAME "Failed to start pod $POD_NAME until timeout"
	
	STARTTIME=$(now)
	ENDTIME=$(( $STARTTIME + $BENCHMARK_DURATION ))

	# Waiting test duration
	$DEBUG && debug "sleeping during the benchmark duration ($BENCHMARK_DURATION) to avoid useless api calls"
	sleep $BENCHMARK_DURATION

	info "Waiting for pod $POD_NAME to be completed"
	# Waiting for test to be succeeded
	waitpod $POD_NAME Succeeded $POD_WAIT_TIMEOUT \
		|| fatal "Failed to run pod $POD_NAME until timeout"

	# Extracting data
	$DEBUG && debug "Writing $POD_NAME logs to $DATADIR/$POD_NAME.log"
	kubectl logs $NAMESPACEOPT $POD_NAME > $DATADIR/$POD_NAME.log
	grep receiver $DATADIR/$POD_NAME.log | $BIN_AWK '{print $7}' > $DATADIR/$POD_NAME.bw
	$DEBUG && debug "$POD_NAME Bandwidth = $(cat $DATADIR/$POD_NAME.bw)"

	# Extracting monitoring
	$DEBUG && debug "Writing $POD_NAME server metrics to $DATADIR/$POD_NAME-server.metrics"
	kubectl exec -it $NAMESPACEOPT $MONITOR_SERVER_POD_NAME \
		-- sh /stats.sh $STARTTIME $ENDTIME > $DATADIR/$POD_NAME-server.metrics

	$DEBUG && debug "Computing $POD_NAME server metrics average to $DATADIR/$POD_NAME-server.avg"
	
	tail -n +2 $DATADIR/$POD_NAME-server.metrics | LC_ALL=C $BIN_AWK '{M+=$6;U+=$1;N+=$2;S+=$3;I+=$4;T+=$5;} 
		END {
			print "cpu-user cpu-nice cpu-system cpu-iowait cpu-steal memory-used records-number"; 
			printf "%.2f %.2f %.2f %.2f %.2f %d %d\n",U/NR,N/NR,S/NR,I/NR,T/NR,M/NR,NR
		}' > $DATADIR/$POD_NAME-server.avg

	$DEBUG && debug "Writing $POD_NAME client metrics to $DATADIR/$POD_NAME-client.metrics"
	kubectl exec -it $NAMESPACEOPT $MONITOR_CLIENT_POD_NAME \
		-- sh /stats.sh $STARTTIME $ENDTIME > $DATADIR/$POD_NAME-client.metrics

	$DEBUG && debug "Computing $POD_NAME client metrics average to $DATADIR/$POD_NAME-client.avg"
	tail -n +2 $DATADIR/$POD_NAME-client.metrics | $BIN_AWK '{M+=$6;U+=$1;N+=$2;S+=$3;I+=$4;T+=$5;} 
		END {
			print "cpu-user cpu-nice cpu-system cpu-iowait cpu-steal memory-used records-number"; 
			printf "%.2f %.2f %.2f %.2f %.2f %d %d\n",U/NR,N/NR,S/NR,I/NR,T/NR,M/NR,NR
		}' > $DATADIR/$POD_NAME-client.avg
}

function compute-data-result {
	(cd $DATADIR; tar -czf - .)
}

function compute-ibdbench-result {
	function compute-ibdbench-metrics {
		POD_NAME=$1
		
		AVGMETRICS=$($BIN_AWK 'NR==2' $DATADIR/$POD_NAME-server.avg)
		echo -en "$($BIN_AWK '{printf $6}' <<< $AVGMETRICS)\t"
		echo -en "$($BIN_AWK '{printf "=%.2f+%.2f+%.2f+%.2f+%.2f",$1,$2,$3,$4,$5}' <<< $AVGMETRICS)\t"

		AVGMETRICS=$($BIN_AWK 'NR==2' $DATADIR/$POD_NAME-client.avg)
		echo -en "$($BIN_AWK '{printf $6}' <<< $AVGMETRICS)\t"
		echo -en "$($BIN_AWK '{printf "=%.2f+%.2f+%.2f+%.2f+%.2f",$1,$2,$3,$4,$5}' <<< $AVGMETRICS)"
	}

	# MTU
	echo -en "$(cat $DATADIR/mtu)\t\t"

	# Idle
	compute-ibdbench-metrics $IDLE_POD_NAME
	echo -en "\t"
	
	# P2P TCP
	echo -en "$(cat $DATADIR/$CLIENT_TCP_P2P_POD_NAME.bw)\t"
	compute-ibdbench-metrics $CLIENT_TCP_P2P_POD_NAME
	echo -en "\t"
	
	# P2P UDP
	echo -en "$(cat $DATADIR/$CLIENT_UDP_P2P_POD_NAME.bw)\t"
	compute-ibdbench-metrics $CLIENT_UDP_P2P_POD_NAME
	echo -en "\t"
	
	# P2S TCP
	echo -en "$(cat $DATADIR/$CLIENT_TCP_P2S_POD_NAME.bw)\t"
	compute-ibdbench-metrics $CLIENT_TCP_P2S_POD_NAME
	echo -en "\t"
	
	# P2S UDP
	echo -en "$(cat $DATADIR/$CLIENT_UDP_P2S_POD_NAME.bw)\t"
	compute-ibdbench-metrics $CLIENT_UDP_P2S_POD_NAME
}

function compute-json-result {
	function compute-json-result-for-pod {
		POD_NAME=$1
		PADDING=$(printf "%${2}s")

		echo "${PADDING}\"bandwidth\": $(cat $DATADIR/$POD_NAME.bw),"
		
		AVGMETRICS=$($BIN_AWK 'NR==2' $DATADIR/$POD_NAME-client.avg)
		$DEBUG && debug "pod $POD_NAME client metrics avg : $AVGMETRICS "
		echo "${PADDING}\"client\": {"
		echo "${PADDING}  \"cpu\": {"
		echo "${PADDING}    \"total\": $($BIN_AWK '{printf "%.2f",$1+$2+$3+$4+$5}' <<< $AVGMETRICS),"
		echo "${PADDING}    \"user\": $($BIN_AWK '{print $1}' <<< $AVGMETRICS),"
		echo "${PADDING}    \"nice\": $($BIN_AWK '{print $2}' <<< $AVGMETRICS),"
		echo "${PADDING}    \"system\": $($BIN_AWK '{print $3}' <<< $AVGMETRICS),"
		echo "${PADDING}    \"iowait\": $($BIN_AWK '{print $4}' <<< $AVGMETRICS),"
		echo "${PADDING}    \"steal\": $($BIN_AWK '{print $5}' <<< $AVGMETRICS)"
		echo "${PADDING}  },"
		echo "${PADDING}  \"ram\": $($BIN_AWK '{print $6}' <<< $AVGMETRICS)"
		echo "${PADDING}},"

		AVGMETRICS=$($BIN_AWK 'NR==2' $DATADIR/$POD_NAME-server.avg)
		$DEBUG && debug "pod $POD_NAME server metrics avg : $AVGMETRICS "
		echo "${PADDING}\"server\": {"
		echo "${PADDING}  \"cpu\": {"
		echo "${PADDING}    \"total\": $($BIN_AWK '{printf "%.2f",$1+$2+$3+$4+$5}' <<< $AVGMETRICS),"
		echo "${PADDING}    \"user\": $($BIN_AWK '{print $1}' <<< $AVGMETRICS),"
		echo "${PADDING}    \"nice\": $($BIN_AWK '{print $2}' <<< $AVGMETRICS),"
		echo "${PADDING}    \"system\": $($BIN_AWK '{print $3}' <<< $AVGMETRICS),"
		echo "${PADDING}    \"iowait\": $($BIN_AWK '{print $4}' <<< $AVGMETRICS),"
		echo "${PADDING}    \"steal\": $($BIN_AWK '{print $5}' <<< $AVGMETRICS)"
		echo "${PADDING}  },"
		echo "${PADDING}  \"ram\": $($BIN_AWK '{print $6}' <<< $AVGMETRICS)"
		echo "${PADDING}}"
	}

	echo "{"
	echo "  \"metadata\": {"
	echo "    \"name\": \"$BENCHMARK_RUN_NAME\","
	echo "    \"generator\": \"$GENERATOR\","
	echo "    \"version\": \"$VERSION\","
	echo "    \"date\": \"$BENCHMARK_DATE\","
	echo "    \"server-node\": \"$SERVER_NODE\","
	echo "    \"client-node\": \"$CLIENT_NODE\""
	echo "    \"socket-buffer-size\": \"$SOCKET_BUFFER_SIZE\""
	echo "  },"
	echo "  \"data\": {"
	echo "    \"cpu\": \"$(cat $DATADIR/cpu)\","
	echo "    \"kernel\": \"$(cat $DATADIR/kernel)\","
	echo "    \"k8s-version\": \"$(cat $DATADIR/k8s-version)\","
	echo "    \"mtu\": \"$(cat $DATADIR/mtu)\","
	if [ "${IDLE_POD_NAME}" != "" ]
	then
		echo "    \"idle\": {"
		compute-json-result-for-pod $IDLE_POD_NAME 6
		echo "    },"
	fi

	if [ "${CLIENT_TCP_P2P_POD_NAME}${CLIENT_UDP_P2P_POD_NAME}" != "" ]
	then
		echo "    \"pod2pod\": {"
		if [ "${CLIENT_TCP_P2P_POD_NAME}" != "" ]
		then
			echo "      \"tcp\": {"
			compute-json-result-for-pod $CLIENT_TCP_P2P_POD_NAME 8
			echo "      },"
		fi
		if [ "${CLIENT_UDP_P2P_POD_NAME}" != "" ]
		then
			echo "      \"udp\": {"
			compute-json-result-for-pod $CLIENT_UDP_P2P_POD_NAME 8
			echo "      }"
		fi
		echo "    },"
	fi

	if [ "${CLIENT_TCP_P2S_POD_NAME}${CLIENT_UDP_P2S_POD_NAME}" != "" ]
	then
		echo "    \"pod2svc\": {"
		if [ "${CLIENT_TCP_P2S_POD_NAME}" != "" ]
		then
			echo "      \"tcp\": {"
			compute-json-result-for-pod $CLIENT_TCP_P2S_POD_NAME 8
			echo "      },"
		fi
		if [ "${CLIENT_UDP_P2S_POD_NAME}" != "" ]
		then
			echo "      \"udp\": {"
			compute-json-result-for-pod $CLIENT_UDP_P2S_POD_NAME 8
			echo "      }"
		fi
		echo "    }"
	fi
	echo "  }"
	echo "}"
}

function compute-yaml-result {

	function compute-yaml-result-for-pod {
		POD_NAME=$1
		PADDING=$(printf "%${2}s")

		echo "${PADDING}bandwidth: $(cat $DATADIR/$POD_NAME.bw)"
		AVGMETRICS=$($BIN_AWK 'NR==2' $DATADIR/$POD_NAME-client.avg)
		$DEBUG && debug "pod $POD_NAME client metrics avg : $AVGMETRICS "
		echo "${PADDING}client:"
		echo "${PADDING}  cpu:"
		echo "${PADDING}    total: $($BIN_AWK '{printf "%.2f",$1+$2+$3+$4+$5}' <<< $AVGMETRICS)"
		echo "${PADDING}    user: $($BIN_AWK '{print $1}' <<< $AVGMETRICS)"
		echo "${PADDING}    nice: $($BIN_AWK '{print $2}' <<< $AVGMETRICS)"
		echo "${PADDING}    system: $($BIN_AWK '{print $3}' <<< $AVGMETRICS)"
		echo "${PADDING}    iowait: $($BIN_AWK '{print $4}' <<< $AVGMETRICS)"
		echo "${PADDING}    steal: $($BIN_AWK '{print $5}' <<< $AVGMETRICS)"
		echo "${PADDING}  ram: $($BIN_AWK '{print $6}' <<< $AVGMETRICS)"

		AVGMETRICS=$($BIN_AWK 'NR==2' $DATADIR/$POD_NAME-server.avg)
		$DEBUG && debug "pod $POD_NAME server metrics avg : $AVGMETRICS "
		echo "${PADDING}server:"
		echo "${PADDING}  cpu:"
		echo "${PADDING}    total: $($BIN_AWK '{printf "%.2f",$1+$2+$3+$4+$5}' <<< $AVGMETRICS)"
		echo "${PADDING}    user: $($BIN_AWK '{print $1}' <<< $AVGMETRICS)"
		echo "${PADDING}    nice: $($BIN_AWK '{print $2}' <<< $AVGMETRICS)"
		echo "${PADDING}    system: $($BIN_AWK '{print $3}' <<< $AVGMETRICS)"
		echo "${PADDING}    iowait: $($BIN_AWK '{print $4}' <<< $AVGMETRICS)"
		echo "${PADDING}    steal: $($BIN_AWK '{print $5}' <<< $AVGMETRICS)"
		echo "${PADDING}  ram: $($BIN_AWK '{print $6}' <<< $AVGMETRICS)"
	}

	echo "metadata:"
	echo "  name: \"$BENCHMARK_RUN_NAME\""
	echo "  generator: \"$GENERATOR\""
	echo "  version: \"$VERSION\""
	echo "  date: \"$BENCHMARK_DATE\""
	echo "  server-node: \"$SERVER_NODE\""
	echo "  client-node: \"$CLIENT_NODE\""
	echo "  socket-buffer-size: \"$SOCKET_BUFFER_SIZE\""
	echo "data:"
	echo "  cpu: \"$(cat $DATADIR/cpu)\""
	echo "  kernel: \"$(cat $DATADIR/kernel)\""
	echo "  k8s-version: \"$(cat $DATADIR/k8s-version)\""
	echo "  mtu: \"$(cat $DATADIR/mtu)\""
	if [ "${IDLE_POD_NAME}" != "" ]
	then
		echo "  idle:"
		compute-yaml-result-for-pod $IDLE_POD_NAME 4
	fi
	if [ "${CLIENT_TCP_P2P_POD_NAME}${CLIENT_UDP_P2P_POD_NAME}" != "" ]
	then
		echo "  pod2pod:"
		if [ "${CLIENT_TCP_P2P_POD_NAME}" != "" ]
		then
			echo "    tcp:"
			compute-yaml-result-for-pod $CLIENT_TCP_P2P_POD_NAME 6
		fi
		if [ "${CLIENT_UDP_P2P_POD_NAME}" != "" ]
		then
			echo "    udp:"
			compute-yaml-result-for-pod $CLIENT_UDP_P2P_POD_NAME 6
		fi
	fi
	if [ "${CLIENT_TCP_P2S_POD_NAME}${CLIENT_UDP_P2S_POD_NAME}" != "" ]
	then
		echo "  pod2svc:"
		if [ "${CLIENT_TCP_P2S_POD_NAME}" != "" ]
		then
			echo "    tcp:"
			compute-yaml-result-for-pod $CLIENT_TCP_P2S_POD_NAME 6
		fi
		if [ "${CLIENT_UDP_P2S_POD_NAME}" != "" ]
		then
			echo "    udp:"
			compute-yaml-result-for-pod $CLIENT_UDP_P2S_POD_NAME 6
		fi
	fi
}

function compute-text-result {
	function compute-text-result-for-pod {
		POD_NAME=$1
		PADDING=$(printf "%${2}s")
		echo "${PADDING}bandwidth = $(cat $DATADIR/$POD_NAME.bw) Mbit/s"
		echo "${PADDING}client cpu = $($BIN_AWK 'NR==2 {printf "total %.2f%% (user %.2f%%, nice %.2f%%, system %.2f%%, iowait %.2f%%, steal %.2f%%)",$1+$2+$3+$4+$5,$1,$2,$3,$4,$5}' $DATADIR/$POD_NAME-client.avg)"
		echo "${PADDING}server cpu = $($BIN_AWK 'NR==2 {printf "total %.2f%% (user %.2f%%, nice %.2f%%, system %.2f%%, iowait %.2f%%, steal %.2f%%)",$1+$2+$3+$4+$5,$1,$2,$3,$4,$5}' $DATADIR/$POD_NAME-server.avg)"
		echo "${PADDING}client ram = $($BIN_AWK 'NR==2 {print $6}' $DATADIR/$POD_NAME-client.avg) MB"
		echo "${PADDING}server ram = $($BIN_AWK 'NR==2 {print $6}' $DATADIR/$POD_NAME-server.avg) MB"
	}

	echo "========================================================="
	echo " Benchmark Results"
	echo "========================================================="
	echo " Name            : $BENCHMARK_RUN_NAME"
	echo " Date            : $BENCHMARK_DATE UTC"
	echo " Generator       : $GENERATOR"
	echo " Version         : $VERSION"
	echo " Server          : $SERVER_NODE"
	echo " Client          : $CLIENT_NODE"
	echo " UDP Socket size : $SOCKET_BUFFER_SIZE"
	echo "========================================================="
	echo "  Discovered CPU         : $(cat $DATADIR/cpu)"
	echo "  Discovered Kernel      : $(cat $DATADIR/kernel)"
	echo "  Discovered k8s version : $(cat $DATADIR/k8s-version)"
	echo "  Discovered MTU         : $(cat $DATADIR/mtu)"
	if [ "${IDLE_POD_NAME}" != "" ]
	then
		echo "  Idle :"
		compute-text-result-for-pod $IDLE_POD_NAME 4
	fi
	if [ "${CLIENT_TCP_P2P_POD_NAME}${CLIENT_UDP_P2P_POD_NAME}" != "" ]
	then
		echo "  Pod to pod :"
		if [ "${CLIENT_TCP_P2P_POD_NAME}" != "" ]
		then
			echo "    TCP :"
			compute-text-result-for-pod $CLIENT_TCP_P2P_POD_NAME 6
		fi
		if [ "${CLIENT_UDP_P2P_POD_NAME}" != "" ]
		then
			echo "    UDP :"
			compute-text-result-for-pod $CLIENT_UDP_P2P_POD_NAME 6
		fi
	fi

	if [ "${CLIENT_TCP_P2S_POD_NAME}${CLIENT_UDP_P2S_POD_NAME}" != "" ]
	then
		echo "  Pod to Service :"
		if [ "${CLIENT_TCP_P2S_POD_NAME}" != "" ]
		then
			echo "    TCP :"; 
			compute-text-result-for-pod $CLIENT_TCP_P2S_POD_NAME 6
		fi
		if [ "${CLIENT_UDP_P2S_POD_NAME}" != "" ]
		then
			echo "    UDP :"
			compute-text-result-for-pod $CLIENT_UDP_P2S_POD_NAME 6
		fi
		echo "========================================================="
	fi
}	

function generate-report {
	case $OUTPUT_FORMAT in
		json) compute-json-result > $FILENAME;;
		yaml) compute-yaml-result > $FILENAME;;
		data) compute-data-result > $FILENAME;;
		ibdbench) compute-ibdbench-result > $FILENAME;;
		*)    compute-text-result > $FILENAME;;
	esac
}

function source-data-from-dir {
	source "$DATADIR/env"
	$DEBUG && debug "GENERATOR=$GENERATOR"
	$DEBUG && debug "VERSION=$VERSION"
	$DEBUG && debug "BENCHMARK_DATE=$BENCHMARK_DATE"
	$DEBUG && debug "SERVER_NODE=$SERVER_NODE"
	$DEBUG && debug "CLIENT_NODE=$CLIENT_NODE"

	$DEBUG && debug "MONITOR_SERVER_POD_NAME=$MONITOR_SERVER_POD_NAME"
	$DEBUG && debug "MONITOR_CLIENT_POD_NAME=$MONITOR_CLIENT_POD_NAME"
	$DEBUG && debug "SERVER_POD_NAME=$SERVER_POD_NAME"
	$DEBUG && debug "IDLE_POD_NAME=$IDLE_POD_NAME"
	$DEBUG && debug "CLIENT_TCP_P2P_POD_NAME=$CLIENT_TCP_P2P_POD_NAME"
	$DEBUG && debug "CLIENT_UDP_P2P_POD_NAME=$CLIENT_UDP_P2P_POD_NAME"
	$DEBUG && debug "CLIENT_TCP_P2S_POD_NAME=$CLIENT_TCP_P2S_POD_NAME"
	$DEBUG && debug "CLIENT_UDP_P2S_POD_NAME=$CLIENT_UDP_P2S_POD_NAME"
}

#==============================================================================
# Argument parsing
#==============================================================================

[ "$1" = "" ] && usage && exit

UNKNOWN_ARGLIST=""
while [ "$1" != "" ]
do
	arg=$1
	case $arg in
		#--- Benchmark mode - Mandatory flags ---------------------------------

		# Define kubernetes node name that will host the client part
		--server-node|-sn)
			shift
			[ "$1" = "" ] && fatal "$arg flag must be followed by a value"
			SERVER_NODE=$1
			info "Server node will be '$SERVER_NODE'"
			;;
		
		# Define kubernetes node name that will host the server part
		--client-node|-cn)
			shift
			[ "$1" = "" ] && fatal "$arg flag must be followed by a value"
			CLIENT_NODE=$1
			info "Client node will be '$CLIENT_NODE'"
			;;

		#--- Benchmark mode - Optionnal flags ---------------------------------
		
		# Set the benchmark duration for each test in seconds
		--duration|-d)
			shift
			[ "$1" = "" ] && fatal "$arg flag must be followed by a value"
			BENCHMARK_DURATION="$1"
			info "Setting benchmark duration to ${1}s"
			;;
		
		# Keep data directory instead of cleaning it (tmp dir that contains raw benchmark data)
		--keep|-k)
			CLEANDIR="false"
			info "Keeping datadir after benchmark run"
			;;
		
		# Set the target kubernetes namespace
		--namespace|-n)
			shift
			[ "$1" = "" ] && fatal "$arg flag must be followed by a value"
			NAMESPACEOPT="--namespace $1"
			info "Setting target namespace to '$1'"
			;;
		
		# Set the name of this benchmark run
		--name)
			shift
			[ "$1" = "" ] && fatal "$arg flag must be followed by a value"
			BENCHMARK_RUN_NAME="$1"
			info "Setting benchmark run nameto '$1'"
			;;

		# Only run a subset of benchmark tests, comma separated (Ex: -ot tcp,idle)
		--only-tests|-ot)
			shift
			[ "$1" = "" ] && fatal "$arg flag must be followed by a value"
			
			RUN_TEST_IDLE="false"
			RUN_TEST_P2P_TCP="false"
			RUN_TEST_P2P_UDP="false"
			RUN_TEST_P2S_TCP="false"
			RUN_TEST_P2S_UDP="false"

			for i in $(awk '{gsub(/,/," ");print}' <<< "$1")
			do
				case $i in
					all)
						RUN_TEST_P2P_TCP="true"
						RUN_TEST_P2P_UDP="true"
						RUN_TEST_P2S_TCP="true"
						RUN_TEST_P2S_UDP="true"
						RUN_TEST_IDLE="true"
						;;
					p2p)
						RUN_TEST_P2P_TCP="true"
						RUN_TEST_P2P_UDP="true"
						;;
					p2s)
						RUN_TEST_P2S_TCP="true"
						RUN_TEST_P2S_UDP="true"
						;;
					tcp)
						RUN_TEST_P2P_TCP="true"
						RUN_TEST_P2S_TCP="true"
						;;
					udp)
						RUN_TEST_P2P_UDP="true"
						RUN_TEST_P2S_UDP="true"
						;;
					p2ptcp) RUN_TEST_P2P_TCP="true" ;;
					p2pudp) RUN_TEST_P2P_UDP="true" ;;
					p2stcp) RUN_TEST_P2S_TCP="true" ;;
					p2sudp) RUN_TEST_P2S_UDP="true" ;;
					idle) RUN_TEST_IDLE="true" ;;
					*) fatal "Unknown test slug '${i}', " ;;
				esac
			done
			info "Benchmark will run only following tests: ${1}"
			;;
		
		# Set the socket buffer size with unit, or 'auto'. ex: '256K' 
		--socket-buffer-size|-sbs)
			shift
			[ "$1" = "" ] && fatal "$arg flag must be followed by a value"
			SOCKET_BUFFER_SIZE="$1"
			info "Setting UDP socket buffer size to ${1}"
			;;

		# Set the pod ready wait timeout in seconds
		--timeout|-t)
			shift
			[ "$1" = "" ] && fatal "$arg flag must be followed by a value"
			POD_WAIT_TIMEOUT="$1"
			info "Setting pod wait timeout to ${1}s"
			;;

		#--- From Data - Mandatory flags --------------------------------------

		# Define the path to the data to read from
		--from-data|-fd)
			shift
			[ "$1" = "" ] && fatal "$arg flag must be followed by a value"
			FROM_DATA="true"
			CLEANDIR="false"
			DATADIR="$1"
			info "Benchmark will not run, report will be generated from ${1}"
			;;
		
		#--- Common - Optionnal flags -----------------------------------------
		
		# Set the debug level to "debug"
		--debug)
			DEBUG_LEVEL=3;
			DEBUG=true;
			debug "Mode debug ON"
			;;

		# Set the debug level 
		--debug-level|-dl)
			shift
			case $1 in
				standard|0) DEBUG_LEVEL=0; DEBUG=false;;
				warn|1) DEBUG_LEVEL=1; DEBUG=false;;
				info|2) DEBUG_LEVEL=2; DEBUG=false;;
				debug|3) DEBUG_LEVEL=3; DEBUG=true; debug "Mode debug ON";;
				*) fatal "Unknown debug level '$1'. Here is a list of acceptable debug levels : standard, warn, info, debug";;
			esac
			;;

		# Set the output file
		--file|-f)
			shift
			[ "$1" = "" ] && fatal "$arg flag must be followed by a value"
			FILENAME="$1"
			[ "$FILENAME" = "-" ] && FILENAME="/dev/stdout"
			info "Output file set to ${FILENAME}"
			;;

		# Display help message
		--help|-h)
			usage && exit
			;;

		# Set the output format
		--output|-o)
			shift
			[ "$1" = "" ] && fatal "$arg flag must be followed by a value"
			# Checking values
			case $1 in
				json) ;;
				yaml) ;;
				text) ;;
				data) ;;
				ibdbench) ;;
				*) fatal "Unknown output format '$1'. Here is a list of acceptable formats : text, json, yaml, data";;
			esac
			
			info "Setting output format to ${1}s"
			OUTPUT_FORMAT="$1"
			;;
		
		# Activate the verbose mode by setting debug-level to 'info'
		--verbose|-v)
			DEBUG_LEVEL=2
			;;

		# Show current script version
		--version|-V)
			echo "$VERSION" 
			exit
			;;
		
		# Unknown flag
		*) UNKNOWN_ARGLIST="$UNKNOWN_ARGLIST $arg" ;;
	esac
	shift
done

[ "$UNKNOWN_ARGLIST" != "" ] && fatal "Unknown arguments : $UNKNOWN_ARGLIST"
$DEBUG && debug "Argument parsing done"

#==============================================================================
# Preflight checks
#==============================================================================

[ "$FILENAME" = "/dev/stdout" -a "$OUTPUT_FORMAT" = "data" ] && fatal "I cowardly refuse to output data on stdout, please use '-f' flag"

#--- Mode detection ---------------------------------------

# We do not run benchmark test, we source from existing data
if $FROM_DATA
then
	$DEBUG && debug "Running in 'from data' mode"

	# Data source is a directory, let's source data
	if [ -d "$DATADIR" -a -f "$DATADIR/env" ]
	then
		$DEBUG && debug "Data is a directory, sourcing env file"

		source-data-from-dir
		generate-report

	# In this case, the datasource is a file, a compressed tar
	elif [ -f "$DATADIR" ]
	then
		$DEBUG && debug "Data is a file"

		# Saving filename
		FROM_DATA_FILE="$DATADIR"
		
		# Generating a temp dir to work
		DATADIR="$(mktemp -d)"

		# Setting the cleandir flag, as datadit is now a temp dir
		CLEANDIR="true"
		
		# Extracting data in DATADIR tmp dir
		tar -xzf "$FROM_DATA_FILE" -C "$DATADIR" || fatal "Cannot extract data from file $FROM_DATA_FILE"

		source-data-from-dir
		generate-report

	else
		fatal "Cannot recognize data format $DATADIR"
	fi

	# In every case, we didn't want to run benchmark test so we exit
	cleanandexit
fi

$DEBUG && debug "Running in 'benchmark' mode"

#--- Node name check --------------------------------------

[ "$SERVER_NODE" = "" -o "$CLIENT_NODE" = "" ] \
	&& fatal "Both client node and server node must be set (--server-node and --client-node)"

#==============================================================================
# Preparation
#==============================================================================

DATADIR="$(mktemp -d)"

# Generating pod names
MONITOR_SERVER_POD_NAME="knb-monitor-server-$EXECID"
MONITOR_CLIENT_POD_NAME="knb-monitor-client-$EXECID"
SERVER_POD_NAME="knb-server-$EXECID"
SERVER_SERVICE_NAME=$SERVER_POD_NAME

IDLE_POD_NAME=""
CLIENT_TCP_P2P_POD_NAME=""
CLIENT_UDP_P2P_POD_NAME=""
CLIENT_TCP_P2S_POD_NAME=""
CLIENT_UDP_P2S_POD_NAME=""

$RUN_TEST_IDLE && IDLE_POD_NAME="knb-client-idle-$EXECID"
$RUN_TEST_P2P_TCP && CLIENT_TCP_P2P_POD_NAME="knb-client-tcp-p2p-$EXECID"
$RUN_TEST_P2P_UDP && CLIENT_UDP_P2P_POD_NAME="knb-client-udp-p2p-$EXECID"
$RUN_TEST_P2S_TCP && CLIENT_TCP_P2S_POD_NAME="knb-client-tcp-p2s-$EXECID"
$RUN_TEST_P2S_UDP && CLIENT_UDP_P2S_POD_NAME="knb-client-udp-p2s-$EXECID"

# Creating env file with metadata
cat <<EOF > $DATADIR/env
BENCHMARK_RUN_NAME="$BENCHMARK_RUN_NAME"
GENERATOR="$GENERATOR"
VERSION="$VERSION"
BENCHMARK_DATE="$BENCHMARK_DATE"
SERVER_NODE="$SERVER_NODE"
CLIENT_NODE="$CLIENT_NODE"

SOCKET_BUFFER_SIZE="$SOCKET_BUFFER_SIZE"

MONITOR_SERVER_POD_NAME="$MONITOR_SERVER_POD_NAME"
MONITOR_CLIENT_POD_NAME="$MONITOR_CLIENT_POD_NAME"
SERVER_POD_NAME="$SERVER_POD_NAME"

IDLE_POD_NAME="$IDLE_POD_NAME"
CLIENT_TCP_P2P_POD_NAME="$CLIENT_TCP_P2P_POD_NAME"
CLIENT_UDP_P2P_POD_NAME="$CLIENT_UDP_P2P_POD_NAME"
CLIENT_TCP_P2S_POD_NAME="$CLIENT_TCP_P2S_POD_NAME"
CLIENT_UDP_P2S_POD_NAME="$CLIENT_UDP_P2S_POD_NAME"
EOF

# Catching CTRL-C
trap ctrl_c INT
function ctrl_c {
        err "Trapped CTRL-C, trying to exit gracefully ..."
		cleanandexit 1
}

#==============================================================================
# Starting monitors
#==============================================================================

#--- Server monitor --------------------------------
info "Deploying server monitor on node $SERVER_NODE"
cat <<EOF | kubectl apply $NAMESPACEOPT -f - >/dev/null|| fatal "Cannot create server monitor pod"
apiVersion: v1
kind: Pod
metadata:
  labels:
    app: $MONITOR_SERVER_POD_NAME
  name: $MONITOR_SERVER_POD_NAME
spec:
  containers:
  - name: monitor
    image: infrabuilder/bench-custom-monitor
  nodeSelector:
    kubernetes.io/hostname: $SERVER_NODE
EOF

RESSOURCE_TO_CLEAN_BEFORE_EXIT="$RESSOURCE_TO_CLEAN_BEFORE_EXIT pod/$MONITOR_SERVER_POD_NAME"

info "Waiting for server monitor to be running"
waitpod $MONITOR_SERVER_POD_NAME Running $POD_WAIT_TIMEOUT \
	|| detect-pod-failure-reason-and-fatal $MONITOR_SERVER_POD_NAME "Failed to start server monitor pod"

#--- Client monitor --------------------------------
info "Deploying client monitor on node $CLIENT_NODE"
cat <<EOF | kubectl apply $NAMESPACEOPT -f - >/dev/null|| fatal "Cannot create client monitor pod"
apiVersion: v1
kind: Pod
metadata:
  labels:
    app: $MONITOR_CLIENT_POD_NAME
  name: $MONITOR_CLIENT_POD_NAME
spec:
  containers:
  - name: monitor
    image: infrabuilder/bench-custom-monitor
  nodeSelector:
    kubernetes.io/hostname: $CLIENT_NODE
EOF

RESSOURCE_TO_CLEAN_BEFORE_EXIT="$RESSOURCE_TO_CLEAN_BEFORE_EXIT pod/$MONITOR_CLIENT_POD_NAME"

info "Waiting for client monitor to be running"
waitpod $MONITOR_CLIENT_POD_NAME Running $POD_WAIT_TIMEOUT \
	|| detect-pod-failure-reason-and-fatal $MONITOR_CLIENT_POD_NAME "Failed to start client monitor pod"

#==============================================================================
# Starting server
#==============================================================================

info "Deploying iperf server on node $SERVER_NODE"
cat <<EOF | kubectl apply $NAMESPACEOPT -f - >/dev/null|| fatal "Cannot create server pod"
apiVersion: v1
kind: Pod
metadata:
  labels:
    app: $SERVER_POD_NAME
  name: $SERVER_POD_NAME
spec:
  containers:
  - name: iperf
    image: infrabuilder/netbench:server-iperf3
    args:
    - iperf3
    - -s
  nodeSelector:
    kubernetes.io/hostname: $SERVER_NODE
---
apiVersion: v1
kind: Service
metadata:
  name: $SERVER_SERVICE_NAME
spec:
  selector:
    app: $SERVER_POD_NAME
  ports:
    - protocol: TCP
      port: 5201
      targetPort: 5201
      name: tcp
    - protocol: UDP
      port: 5201
      targetPort: 5201
      name: udp
EOF

RESSOURCE_TO_CLEAN_BEFORE_EXIT="$RESSOURCE_TO_CLEAN_BEFORE_EXIT pod/$SERVER_POD_NAME"

info "Waiting for server to be running"
kubectl wait $NAMESPACEOPT --for=condition=Ready pod/$SERVER_POD_NAME --timeout=${POD_WAIT_TIMEOUT}s >/dev/null \
	|| detect-pod-failure-reason-and-fatal $SERVER_POD_NAME "Failed to start server pod"

SERVER_IP=$(kubectl get $NAMESPACEOPT pod $SERVER_POD_NAME -o jsonpath={.status.podIP})
$DEBUG && debug "Server IP address is $SERVER_IP"

#==============================================================================
# Data Detection
#==============================================================================

#--- CPU Detection ----------------------------------------

info "Detecting CPU"
DISCOVERED_CPU=$(kubectl exec -it  $NAMESPACEOPT $SERVER_POD_NAME -- grep "model name" /proc/cpuinfo | tail -n 1 | awk -F: '{print $2}')
$DEBUG && debug "Cpu is $DISCOVERED_CPU"
echo $DISCOVERED_CPU > $DATADIR/cpu

#--- Kernel Detection -------------------------------------

info "Detecting Kernel"
DISCOVERED_KERNEL=$(kubectl exec -it  $NAMESPACEOPT $SERVER_POD_NAME -- uname -r)
$DEBUG && debug "Kernel is $DISCOVERED_KERNEL"
echo $DISCOVERED_KERNEL > $DATADIR/kernel


#--- K8S Version Detection --------------------------------

info "Detecting Kubernetes version"
DISCOVERED_K8S_VERSION=$(kubectl version --short | awk '$1=="Server" {print $3}' )
$DEBUG && debug "Discovered k8s version is $DISCOVERED_K8S_VERSION"
echo $DISCOVERED_K8S_VERSION > $DATADIR/k8s-version

#--- MTU Detection ----------------------------------------

info "Detecting CNI MTU"
CNI_MTU=$(kubectl exec -it  $NAMESPACEOPT $SERVER_POD_NAME -- ip link \
	| grep "UP,LOWER_UP" | grep -v LOOPBACK | grep -oE "mtu [0-9]*"| $BIN_AWK '{print $2}')
$DEBUG && debug "CNI MTU is $CNI_MTU"
echo $CNI_MTU > $DATADIR/mtu

#==============================================================================
# Benchmark tests
#==============================================================================

#--- Idle measurment  -------------------------------------

if [ "$IDLE_POD_NAME" != "" ]
then
	run-client $IDLE_POD_NAME $SERVER_IP idle
else
	info "Skipping Idle test"
fi

#--- Pod to pod -------------------------------------------

if [ "$CLIENT_TCP_P2P_POD_NAME" != "" ]
then
	run-client $CLIENT_TCP_P2P_POD_NAME $SERVER_IP tcp
else
	info "Skipping P2P TCP test"
fi

if [ "$CLIENT_UDP_P2P_POD_NAME" != "" ]
then
	run-client $CLIENT_UDP_P2P_POD_NAME $SERVER_IP udp
else
	info "Skipping P2P UDP test"
fi

#--- Pod to Service ---------------------------------------

if [ "$CLIENT_TCP_P2S_POD_NAME" != "" ]
then
	run-client $CLIENT_TCP_P2S_POD_NAME $SERVER_SERVICE_NAME tcp
else
	info "Skipping P2S TCP test"
fi

if [ "$CLIENT_UDP_P2S_POD_NAME" != "" ]
then
	run-client $CLIENT_UDP_P2S_POD_NAME $SERVER_SERVICE_NAME udp
else
	info "Skipping P2S UDP test"
fi

#==============================================================================
# Output
#==============================================================================

generate-report

#==============================================================================
# Cleaning
#==============================================================================

info "Cleaning kubernetes resources ..."
cleanandexit